﻿using System;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Text.RegularExpressions;

namespace OrandoLabs.OpenSource.Common
{
    public class TextTokenizer
    {
        static String EnglishWordsAndNumbers = "(\\p{L}+['\x2019]?\\p{L}+)|(\\p{N}+[.]?\\p{N}+)";

        TextReader Reader;
        List<String> Tokens = new List<String>();

        public TextTokenizer(TextReader reader)
        {
            Reader = reader;
        }

        public String PopToken()
        {
            String token = Tokens.First();
            Tokens.RemoveAt(0);
            return token;
        }

        String Whitespace = "[\\p{Z}\\p{C} ]+";
        public async Task<String> NextToken()
        {
            if (Tokens.Count > 0)
                return PopToken();

            while (true)
            {
                String buffer = await NextBuffer();
                if (buffer.Length == 0) 
                    return "";
                ProcessBuffer(buffer);
                if (Tokens.Count > 0)
                    return PopToken();
            }
        }

        protected virtual void ProcessBuffer(String buffer)
        {
            String[] split = Regex.Split(buffer, Whitespace);
            ProcessRuns(split);
        }

        protected void ProcessRuns(String[] runs)
        {
            foreach (String run in runs)
                MatchRuns(run);
        }

        protected virtual void MatchRuns(String run)
        {
            Tokens.AddRange(Match(run));
        }

        protected virtual IEnumerable<String> Match(String run)
        {
            var matches = Regex.Matches(run, EnglishWordsAndNumbers);
            List<String> tokens = new List<String>();
            foreach (Match match in matches)
                tokens.Add(match.Value);
            return tokens;
        }

        int BufferRemaining = 0;
        char[] Buffer = new char[2048];
        async Task<String> NextBuffer()
        {
            StringBuilder buffer = new StringBuilder();
            int len = await Reader.ReadAsync(Buffer, BufferRemaining, Buffer.Length - BufferRemaining);
            if (len <= 0)
            {
                if (BufferRemaining > 0)
                {
                    buffer.Append(Buffer, 0, BufferRemaining);
                    BufferRemaining = 0;
                    return buffer.ToString();
                }
                else
                    return "";
            }
            else
            {
                int buffLen = len + BufferRemaining;
                buffer.Append(Buffer, 0, buffLen);
                BufferRemaining = 0;
                if (IsWhiteSpace(Buffer[buffLen - 1]))
                    return buffer.ToString();
            }

            while (true)
            {
                len = await Reader.ReadAsync(Buffer, 0, Buffer.Length);
                if (len <= 0)
                    return buffer.ToString();

                for (int i = len - 1; i >= 0; i--)
                {
                    if (!IsWhiteSpace(buffer[i]))
                        continue;

                    buffer.Append(Buffer, 0, i);
                    BufferRemaining = len - i - 1;
                    for (int j = 0; j < BufferRemaining; j++)
                        Buffer[j] = Buffer[BufferRemaining + j];
                    return buffer.ToString();
                }
            }

        }

        bool IsWhiteSpace(char ch)
        {
            return Regex.IsMatch(new String(ch, 1), Whitespace);
        }

    }

}
